{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b50e216-6915-4644-b4c7-434a6fa2f843",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --quiet llama-index-core llama-index-utils-workflow llama-index-llms-openai llama-index-graph-stores-neo4j "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5e873d91-0953-483c-ad27-922986b36ad0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pandas/core/arrays/masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from llama_index.llms.openai import OpenAI\n",
    "from neo4j.exceptions import CypherSyntaxError\n",
    "\n",
    "from llama_index.core import ChatPromptTemplate\n",
    "from pydantic import BaseModel, Field\n",
    "from typing import List, Literal, Union, Optional\n",
    "\n",
    "from llama_index.core.workflow import (\n",
    "    StartEvent,\n",
    "    StopEvent,\n",
    "    Workflow,\n",
    "    step,\n",
    "    Event,\n",
    "    Context,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7012b2a4-de4f-4a64-9945-8f68a79870d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.graph_stores.neo4j import Neo4jPropertyGraphStore\n",
    "\n",
    "graph_store = Neo4jPropertyGraphStore(\n",
    "    username=\"recommendations\",\n",
    "    password=\"recommendations\",\n",
    "    database=\"recommendations\",\n",
    "    url=\"neo4j+s://demo.neo4jlabs.com:7687\",\n",
    "    enhanced_schema=True,\n",
    "    create_indexes=False,\n",
    "    timeout=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a18727dc-376b-4f76-a052-7ad57ae1a182",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nllm = OpenAI(model=\"gpt-4o-2024-11-20\", temperature=0)\\nllm_fast = OpenAI(model=\"gpt-4o-2024-11-20\", temperature=0)\\n'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-\"\n",
    "\"\"\"\n",
    "llm = OpenAI(model=\"gpt-4o-2024-11-20\", temperature=0)\n",
    "llm_fast = OpenAI(model=\"gpt-4o-2024-11-20\", temperature=0)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d0def5b4-1812-4729-bc10-6e1466578067",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.gemini import Gemini\n",
    "\n",
    "llm = Gemini(\n",
    "    model=\"models/gemini-2.0-flash-exp\",\n",
    "    api_key=\"\"\n",
    ")\n",
    "\n",
    "llm_fast = Gemini(\n",
    "    model=\"models/gemini-1.5-flash\",\n",
    "    api_key=\"\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2cf99343-8559-4f98-ab8b-355febf59649",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Guardrail(BaseModel):\n",
    "    \"\"\"Guardrail\"\"\"\n",
    "\n",
    "    decision: Literal[\"movie\", \"end\"] = Field(\n",
    "        description=\"Decision on whether the question is related to movies\"\n",
    "        \n",
    "    )\n",
    "guardrails_system_prompt = \"\"\"As an intelligent assistant, your primary objective is to decide whether a given question is related to movies or not.\n",
    "If the question is related to movies, output \"movie\". Otherwise, output \"end\".\n",
    "To make this decision, assess the content of the question and determine if it refers to any movie, actor, director, film industry,\n",
    "or related topics. Provide only the specified output: \"movie\" or \"end\".\"\"\"\n",
    "# Refine Prompt\n",
    "chat_refine_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        guardrails_system_prompt,\n",
    "    ),\n",
    "    (\"user\", \"The question is: {question}\"),\n",
    "]\n",
    "guardrails_template = ChatPromptTemplate.from_messages(chat_refine_msgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e618e42c-6ee1-434e-be80-986b5bfe607f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SubqueriesOutput(BaseModel):\n",
    "    \"\"\"Defines the output format for transforming a question into parallel-optimized retrieval steps.\"\"\"\n",
    "    \n",
    "    plan: List[List[str]] = Field(description=(\"\"\"A list of query groups where:\n",
    "        - Each group (inner list) contains queries that can be executed in parallel\n",
    "        - Groups are ordered by dependency (earlier groups must be executed before later ones)\n",
    "        - Each query must be a specific information retrieval request\n",
    "        - No reasoning or comparison tasks, only data fetching queries\"\"\"))\n",
    "\n",
    "subqueries_system = \"\"\"You are a query planning optimizer. Your task is to break down complex questions into efficient, parallel-optimized retrieval steps. Focus ONLY on information retrieval queries, not analysis or reasoning steps.\n",
    "\n",
    "Key Requirements:\n",
    "- Group queries that can be executed in parallel into the same list\n",
    "- Order groups based on data dependencies\n",
    "- Include ONLY specific information retrieval queries\n",
    "- Exclude reasoning tasks, comparisons, or analysis steps\n",
    "- Prioritize queries that can be executed first and in parallel\n",
    "\n",
    "For simple, directly answerable questions, return a single query in a single group.\n",
    "\n",
    "Example 1:\n",
    "User: \"What was the impact of the 2008 financial crisis on Bank of America's stock price and employee count?\"\n",
    "Assistant: [\n",
    "    # Group 1 - These can be fetched in parallel\n",
    "    [\n",
    "        \"What was Bank of America's stock price history from 2007 to 2009?\",\n",
    "        \"What was Bank of America's total employee count in 2007?\",\n",
    "        \"What was Bank of America's total employee count in 2009?\"\n",
    "    ]\n",
    "]\n",
    "\n",
    "Example 2:\n",
    "User: \"Compare the performance of Tesla's Model 3 with BMW's competing models in terms of range and acceleration.\"\n",
    "Assistant: [\n",
    "    # Group 1 - Basic specs can be fetched in parallel\n",
    "    [\n",
    "        \"What is the EPA range of the Tesla Model 3?\",\n",
    "        \"What is the 0-60 mph acceleration time of the Tesla Model 3?\",\n",
    "        \"What BMW models compete directly with the Tesla Model 3?\"\n",
    "    ],\n",
    "    # Group 2 - Depends on knowing competing models from group 1\n",
    "    [\n",
    "        \"What is the EPA range of each identified BMW competitor model?\",\n",
    "        \"What is the 0-60 mph acceleration time of each identified BMW competitor model?\"\n",
    "    ]\n",
    "]\n",
    "\n",
    "Remember:\n",
    "- Focus on data retrieval only\n",
    "- Maximize parallel execution opportunities\n",
    "- Maintain necessary sequential ordering\n",
    "- Keep queries specific and self-contained\n",
    "- Prioritize independent queries first\"\"\"\n",
    "\n",
    "query_decompose_msgs = [\n",
    "    (\"system\", subqueries_system),\n",
    "    (\"user\", \"{question}\")\n",
    "]\n",
    "\n",
    "subquery_template = ChatPromptTemplate.from_messages(query_decompose_msgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbf05240-d619-499c-83fe-dd1f917c55dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def guardrails_step(question):\n",
    "    guardrails_output = (\n",
    "        llm_fast.as_structured_llm(Guardrail)\n",
    "        .complete(guardrails_template.format(question=question))\n",
    "        .raw\n",
    "    ).decision\n",
    "    if guardrails_output == 'end':\n",
    "        context = \"The question is not about movies or their case, so I cannot answer this question\"\n",
    "        return {\"next_event\": \"generate_final_answer\", \"arguments\": {\"context\": context, \"question\": question}}\n",
    "    # Refactor into separate step\n",
    "    queries_output = (\n",
    "        llm.as_structured_llm(SubqueriesOutput)\n",
    "        .complete(subquery_template.format(question=question))\n",
    "        .raw\n",
    "    ).plan\n",
    "    return {\"next_event\": \"generate_cypher\", \"arguments\": {\"plan\": queries_output, \"question\": question}}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fd6844fa-f5ad-42c2-8312-43541410b7e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_fields.py:132: UserWarning: Field \"model_name\" in HuggingFaceInferenceAPIEmbeddings has conflict with protected namespace \"model_\".\n",
      "\n",
      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from llama_index.core.schema import TextNode\n",
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
    "\n",
    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "\n",
    "\n",
    "examples = [\n",
    "    {\n",
    "        \"question\": \"How many artists are there?\",\n",
    "        \"query\": \"MATCH (a:Person)-[:ACTED_IN]->(:Movie) RETURN count(DISTINCT a)\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"Which actors played in the movie Casino?\",\n",
    "        \"query\": \"MATCH (m:Movie {title: 'Casino'})<-[:ACTED_IN]-(a) RETURN a.name\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"How many movies has Tom Hanks acted in?\",\n",
    "        \"query\": \"MATCH (a:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(m:Movie) RETURN count(m)\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"List all the genres of the movie Schindler's List\",\n",
    "        \"query\": \"MATCH (m:Movie {title: 'Schindler's List'})-[:IN_GENRE]->(g:Genre) RETURN g.name\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"Which actors have worked in movies from both the comedy and action genres?\",\n",
    "        \"query\": \"MATCH (a:Person)-[:ACTED_IN]->(:Movie)-[:IN_GENRE]->(g1:Genre), (a)-[:ACTED_IN]->(:Movie)-[:IN_GENRE]->(g2:Genre) WHERE g1.name = 'Comedy' AND g2.name = 'Action' RETURN DISTINCT a.name\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"Which directors have made movies with at least three different actors named 'John'?\",\n",
    "        \"query\": \"MATCH (d:Person)-[:DIRECTED]->(m:Movie)<-[:ACTED_IN]-(a:Person) WHERE a.name STARTS WITH 'John' WITH d, COUNT(DISTINCT a) AS JohnsCount WHERE JohnsCount >= 3 RETURN d.name\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"Identify movies where directors also played a role in the film.\",\n",
    "        \"query\": \"MATCH (p:Person)-[:DIRECTED]->(m:Movie), (p)-[:ACTED_IN]->(m) RETURN m.title, p.name\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"Find the actor with the highest number of movies in the database.\",\n",
    "        \"query\": \"MATCH (a:Actor)-[:ACTED_IN]->(m:Movie) RETURN a.name, COUNT(m) AS movieCount ORDER BY movieCount DESC LIMIT 1\",\n",
    "    },\n",
    "]\n",
    "\n",
    "few_shot_nodes = []\n",
    "for line in examples:\n",
    "    few_shot_nodes.append(TextNode(text=f\"{{'query':{line['query']}, 'question': {line['question']}))\"))\n",
    "\n",
    "few_shot_index = VectorStoreIndex(few_shot_nodes, embed_model=embed_model)\n",
    "few_shot_retriever = few_shot_index.as_retriever(similarity_top_k=5)\n",
    "\n",
    "\n",
    "def get_fewshots(question):\n",
    "    return [el.text for el in few_shot_retriever.retrieve(question)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c0597674-4e7d-4723-8094-3f1adeee3ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_system = \"\"\"Given an input question, convert it to a Cypher query. No pre-amble.\n",
    "Do not wrap the response in any backticks or anything else. Respond with a Cypher statement only!\"\"\"\n",
    "\n",
    "generate_user = \"\"\"You are a Neo4j expert. Given an input question, create a syntactically correct Cypher query to run.\n",
    "Do not wrap the response in any backticks or anything else. Respond with a Cypher statement only!\n",
    "Here is the schema information\n",
    "{schema}\n",
    "\n",
    "Below are a number of examples of questions and their corresponding Cypher queries.\n",
    "\n",
    "{fewshot_examples}\n",
    "\n",
    "User input: {question}\n",
    "Cypher query:\"\"\"\n",
    "\n",
    "generate_cypher_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        generate_system,\n",
    "    ),\n",
    "    (\"user\", generate_user),\n",
    "]\n",
    "\n",
    "text2cypher_prompt = ChatPromptTemplate.from_messages(generate_cypher_msgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4ec54fde-6995-4823-9150-b9798763d504",
   "metadata": {},
   "outputs": [],
   "source": [
    "schema = graph_store.get_schema_str(exclude_types=[\"Actor\", \"Director\"])\n",
    "\n",
    "async def generate_cypher(subquery):\n",
    "    fewshot_examples = get_fewshots(subquery)\n",
    "    resp = await llm.achat(text2cypher_prompt.format_messages(question=subquery, schema=schema, fewshot_examples=fewshot_examples))\n",
    "    return resp.message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7804a532-64a0-41dc-830b-c50bf75d75fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_cypher_system = \"\"\"You are a specialized parser focused on analyzing Cypher query statements to extract node property filters. Your task is to identify and extract properties used in WHERE clauses and pattern matching conditions, but only when they contain explicit literal values.\n",
    "\n",
    "For each Cypher statement, you should:\n",
    "\n",
    "1. Identify node labels and their associated property filters\n",
    "2. Extract the property key and its matching literal value\n",
    "3. Format the output as a JSON object containing a \"filters\" array with objects having:\n",
    "   - node_label: The label of the node (e.g., \"Person\")\n",
    "   - property_key: The property name being filtered (e.g., \"age\")\n",
    "   - property_value: The literal value being matched (e.g., 30)\n",
    "\n",
    "Rules for extraction:\n",
    "- Only extract filters that match against literal values (strings, numbers, booleans)\n",
    "- Include property filters in MATCH patterns (e.g., (p:Person {name: 'John'}))\n",
    "- Include property filters in WHERE clauses with literal values (e.g., WHERE p.age > 30)\n",
    "- Handle both simple equality and comparison operators with literal values\n",
    "- Ignore property-to-property comparisons (e.g., WHERE m1.rating = m2.rating)\n",
    "- Ignore variable references or dynamic values\n",
    "- Valid literal values are:\n",
    "  * Quoted strings (e.g., 'John', \"London\")\n",
    "  * Numbers (e.g., 30, 42.5)\n",
    "  * Booleans (true, false)\n",
    "\n",
    "Example input 1:\n",
    "MATCH (p:Person {name: 'John'})-[:KNOWS]->(f:Friend)\n",
    "WHERE p.age > 30 AND f.city = 'London' AND f.salary = m.salary\n",
    "\n",
    "Example output 1:\n",
    "{\n",
    "    \"filters\": [\n",
    "        {\n",
    "            \"node_label\": \"Person\",\n",
    "            \"property_key\": \"name\",\n",
    "            \"property_value\": \"John\"\n",
    "        },\n",
    "        {\n",
    "            \"node_label\": \"Person\",\n",
    "            \"property_key\": \"age\",\n",
    "            \"property_value\": 30\n",
    "        },\n",
    "        {\n",
    "            \"node_label\": \"Friend\",\n",
    "            \"property_key\": \"city\",\n",
    "            \"property_value\": \"London\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "Example input 2:\n",
    "MATCH (m1:Movie {title: 'Matrix'}), (m2:Movie)\n",
    "WHERE m1.rating > m2.rating AND m1.year = 1999\n",
    "\n",
    "Example output 2:\n",
    "{\n",
    "    \"filters\": [\n",
    "        {\n",
    "            \"node_label\": \"Movie\",\n",
    "            \"property_key\": \"title\",\n",
    "            \"property_value\": \"Matrix\"\n",
    "        },\n",
    "        {\n",
    "            \"node_label\": \"Movie\",\n",
    "            \"property_key\": \"year\",\n",
    "            \"property_value\": 1999\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "Note how property-to-property comparisons (f.salary = m.salary, m1.rating > m2.rating) are ignored in the output.\"\"\"\n",
    "\n",
    "validate_cypher_user = \"\"\"Cypher statement: {cypher}\"\"\"\n",
    "\n",
    "validate_cypher_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        validate_cypher_system,\n",
    "    ),\n",
    "    (\"user\", validate_cypher_user),\n",
    "]\n",
    "\n",
    "validate_cypher_prompt = ChatPromptTemplate.from_messages(validate_cypher_msgs)\n",
    "\n",
    "class Property(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents a filter condition based on a specific node property in a graph in a Cypher statement.\n",
    "    \"\"\"\n",
    "\n",
    "    node_label: str = Field(\n",
    "        description=\"The label of the node to which this property belongs.\"\n",
    "    )\n",
    "    property_key: str = Field(description=\"The key of the property being filtered.\")\n",
    "    property_value: str = Field(\n",
    "        description=\"The value that the property is being matched against.\"\n",
    "    )\n",
    "\n",
    "\n",
    "class ValidateCypherOutput(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents the applied filters of a Cypher query's output.\n",
    "    \"\"\"\n",
    "    filters: Optional[List[Property]] = Field(\n",
    "        description=\"A list of property-based filters applied in the Cypher statement.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7795a143-f6e5-4198-bd9d-3efc1d98a036",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.graph_stores.neo4j import CypherQueryCorrector, Schema\n",
    "\n",
    "# Cypher query corrector is experimental\n",
    "corrector_schema = [\n",
    "    Schema(el[\"start\"], el[\"type\"], el[\"end\"])\n",
    "    for el in graph_store.get_schema().get(\"relationships\")\n",
    "]\n",
    "cypher_query_corrector = CypherQueryCorrector(corrector_schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ba0a4ea0-63be-4309-b5ce-75dd9229a4bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_cypher(question, cypher):\n",
    "    \"\"\"\n",
    "    Validates the Cypher statements and maps any property values to the database.\n",
    "    \"\"\"\n",
    "    errors = []\n",
    "    mapping_errors = []\n",
    "    # Check for syntax errors\n",
    "    try:\n",
    "        graph_store.structured_query(f\"EXPLAIN {cypher}\")\n",
    "    except CypherSyntaxError as e:\n",
    "        errors.append(e.message)\n",
    "    # Experimental feature for correcting relationship directions\n",
    "    corrected_cypher = cypher_query_corrector(cypher)\n",
    "    if not corrected_cypher:\n",
    "        errors.append(\"The generated Cypher statement doesn't fit the graph schema\")\n",
    "    # Use LLM for mapping for values\n",
    "    llm_output =   (\n",
    "        llm.as_structured_llm(ValidateCypherOutput)\n",
    "        .complete(validate_cypher_prompt.format(cypher=cypher))\n",
    "        .raw\n",
    "    )\n",
    "    if llm_output.filters:\n",
    "        for filter in llm_output.filters:\n",
    "            # Do mapping only for string values\n",
    "            try:\n",
    "                if (\n",
    "                    not [\n",
    "                        prop\n",
    "                        for prop in graph_store.get_schema()[\"node_props\"][\n",
    "                            filter.node_label\n",
    "                        ]\n",
    "                        if prop[\"property\"] == filter.property_key\n",
    "                    ][0][\"type\"]\n",
    "                    == \"STRING\"\n",
    "                ):\n",
    "                    continue\n",
    "            except: # if property is hallucinated/doesn't exist in the schema # ToDo handle it better\n",
    "                continue\n",
    "            mapping = graph_store.structured_query(\n",
    "                f\"MATCH (n:{filter.node_label}) WHERE toLower(n.`{filter.property_key}`) = toLower($value) RETURN 'yes' LIMIT 1\",\n",
    "                {\"value\": filter.property_value},\n",
    "            )\n",
    "            if not mapping:\n",
    "                mapping_errors.append(\n",
    "                    f\"Could not find node in graph with label '{filter.node_label}' where property '{filter.property_key}' equals '{filter.property_value}'. \"\n",
    "                    f\"Without this information, I cannot provide a complete answer to your question. \"\n",
    "                    f\"If you meant something else, please rephrase your question or verify the specific {filter.property_key} you're asking about. \"\n",
    "                    f\"Would you like to try with a different {filter.property_key} value?\"\n",
    "                )\n",
    "    if mapping_errors:\n",
    "        next_action = \"end\"\n",
    "    elif errors:\n",
    "        next_action = \"correct_cypher\"\n",
    "    else:\n",
    "        next_action = \"execute_cypher\"\n",
    "\n",
    "    return {\n",
    "        \"next_action\": next_action,\n",
    "        \"cypher_statement\": corrected_cypher,\n",
    "        \"cypher_errors\": errors,\n",
    "        \"mapping_errors\": mapping_errors,\n",
    "        \"steps\": [\"validate_cypher\"],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "44d23e86-bbba-42c4-97f7-412a6a93ac25",
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_cypher_system = \"\"\"You are a Cypher expert reviewing a statement written by a junior developer. \n",
    "You need to correct the Cypher statement based on the provided errors. No pre-amble.\"\n",
    "Do not wrap the response in any backticks or anything else. Respond with a Cypher statement only!\"\"\"\n",
    "\n",
    "correct_cypher_user = \"\"\"Check for invalid syntax or semantics and return a corrected Cypher statement.\n",
    "\n",
    "Schema:\n",
    "{schema}\n",
    "\n",
    "Note: Do not include any explanations or apologies in your responses.\n",
    "Do not wrap the response in any backticks or anything else.\n",
    "Respond with a Cypher statement only!\n",
    "\n",
    "Do not respond to any questions that might ask anything else than for you to construct a Cypher statement.\n",
    "\n",
    "The question is:\n",
    "{question}\n",
    "\n",
    "The Cypher statement is:\n",
    "{cypher}\n",
    "\n",
    "The errors are:\n",
    "{errors}\n",
    "\n",
    "Corrected Cypher statement: \"\"\"\n",
    "\n",
    "# Correct cypher\n",
    "correct_cypher_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        correct_cypher_system,\n",
    "    ),\n",
    "    (\"user\", correct_cypher_user),\n",
    "]\n",
    "\n",
    "correct_cypher_prompt = ChatPromptTemplate.from_messages(correct_cypher_msgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9dfe2255-8146-4082-8ada-78b5ff95a79c",
   "metadata": {},
   "outputs": [],
   "source": [
    "async def correct_cypher(subquery, cypher, errors):\n",
    "    resp = await llm_fast.achat(correct_cypher_prompt.format_messages(question=subquery, schema=schema, errors=errors))\n",
    "    return resp.message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1b2562aa-64ec-42c8-b1f4-dfa285c26751",
   "metadata": {},
   "outputs": [],
   "source": [
    "information_check_system = \"\"\"You are an expert assistant that evaluates whether a set of subqueries, their results, any existing condensed information, and the current query plan provide enough details to answer a given question. Your task is to:\n",
    "\n",
    "1. Analyze if the available information is sufficient to answer the original question: \"{original_question}\".\n",
    "2. Review the remaining steps in the query plan (if any) to determine if they:\n",
    "   - Should be retained as is.\n",
    "   - Need modification to address gaps.\n",
    "   - Can be skipped because the required information is already available.\n",
    "   - Should be reorganized for optimized execution (e.g., parallel processing).\n",
    "\n",
    "### Process:\n",
    "\n",
    "#### 1. Analyze the Original Question\n",
    "   - Identify the key pieces of information required to fully answer the question.\n",
    "   - Break the question into smaller components if it involves multiple aspects.\n",
    "\n",
    "#### 2. Review Available Information\n",
    "   - Examine the subqueries, their results, and any provided condensed information.\n",
    "   - Assess if they collectively address all components of the question.\n",
    "\n",
    "#### 3. Identify Information Gaps\n",
    "   - Compare the requirements from the question against the available information.\n",
    "   - Highlight any missing details or incomplete data that must be retrieved to form a complete answer.\n",
    "   - If any critical subqueries fail to produce results essential for answering the question:\n",
    "     - Document the missing information in the **dynamic notebook**.\n",
    "     - Mark the task as **unsolvable** due to the missing data.\n",
    "     - Do not produce a modified query plan, as the question cannot be answered with the available or fetchable information.\n",
    "\n",
    "#### 4. Update and Refine the **Dynamic Notebook**\n",
    "   - Treat the condensed information as a **central knowledge base**:\n",
    "     - Continuously update it with key details from subquery results.\n",
    "     - Integrate new data to close gaps and establish connections between facts.\n",
    "   - Ensure the notebook reflects the current state of knowledge, including any identified gaps or failed subqueries.\n",
    "   - Document explicitly if the task is unsolvable due to missing information.\n",
    "\n",
    "#### 5. Modify the Query Plan (If Solvable)\n",
    "   - If sufficient information is available:\n",
    "     - Generate a concise and accurate answer to the question.\n",
    "     - Specify which remaining steps (if any) in the query plan can be skipped.\n",
    "   - If information is insufficient but fetchable:\n",
    "     - Suggest additional subqueries to retrieve the missing details.\n",
    "     - Ensure new subqueries are designed specifically to fill identified gaps.\n",
    "     - Organize all subqueries into parallel-executable groups wherever possible.\n",
    "     - Maintain sequential steps only when strict data dependencies exist.\n",
    "   - If critical gaps exist that cannot be resolved due to failed subqueries, do not modify the query plan and clearly state why the task cannot be completed.\n",
    "\n",
    "### Key Guidelines:\n",
    "- **Focus Only on Information Retrieval**: Limit query plans to fetching data and avoid reasoning/analysis tasks.\n",
    "- **Optimize for Parallel Execution**: Group queries into parallelizable blocks to reduce execution time.\n",
    "- **Maintain Sequential Order Only When Necessary**: Use sequential steps only when results of one query depend on another.\n",
    "- **Centralize Knowledge**:\n",
    "   - Use the dynamic notebook to consolidate all available information.\n",
    "   - Ensure it remains the authoritative source for answering the question and guiding further steps.\n",
    "   - Explicitly document unsolvable tasks if critical information is missing.\n",
    "\"\"\"\n",
    "\n",
    "information_check_user = \"\"\"\n",
    "Subqueries and their results:  \n",
    "{subqueries}  \n",
    "Existing dynamic notebook:  \n",
    "{dynamic_notebook}\n",
    "Current remaining plan (if any):\n",
    "{plan}\n",
    "Original question: {question}  \n",
    "\"\"\"\n",
    "\n",
    "information_check_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        information_check_system,\n",
    "    ),\n",
    "    (\"user\", information_check_user),\n",
    "]\n",
    "\n",
    "information_check_prompt = ChatPromptTemplate.from_messages(information_check_msgs)\n",
    "\n",
    "class IFOutput(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents the output of an information sufficiency evaluation process. \n",
    "    Contains either a condensed summary of the available information or additional subqueries needed to answer the original question.\n",
    "    \"\"\"\n",
    "\n",
    "    dynamic_notebook: str = Field(\n",
    "        description=\"A continuously updated and refined summary integrating subquery results and condensed information. Serves as the central knowledge base to address the original question and guide further subqueries if necessary.\"\n",
    "    )\n",
    "    modified_plan: Optional[List[List[str]]] = Field(\n",
    "        description=\"Modified version of the remaining plan steps. Each group contains queries that can be executed in parallel. Null if no remaining plan exists, all gaps have been addressed, or the task is unsolvable due to missing critical information.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "15837b79-6416-425d-8eb3-7e1d91af267c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_subqueries_for_prompt(information_checks: list) -> str:\n",
    "    \"\"\"\n",
    "    Converts a list of InformationCheck objects into a string that can be added to a prompt.\n",
    "    \n",
    "    Args:\n",
    "        information_checks (List[InformationCheck]): List of information checks to process.\n",
    "    \n",
    "    Returns:\n",
    "        str: A formatted string representing subqueries and their results.\n",
    "    \"\"\"\n",
    "    subqueries_and_results = []\n",
    "    \n",
    "    for check in information_checks:\n",
    "        # Extract the first result if available, otherwise use \"No result available.\"\n",
    "        result = (\n",
    "            check.database_output[0] if check.database_output else \"No result available.\"\n",
    "        )\n",
    "        subqueries_and_results.append(\n",
    "            f\"- Subquery: {check.subquery}\\n  Result: {result}\"\n",
    "        )\n",
    "    \n",
    "    return \"\\n\".join(subqueries_and_results)\n",
    "\n",
    "def information_check(subquery_events, original_question, dynamic_notebook, plan):\n",
    "    subqueries = format_subqueries_for_prompt(subquery_events)\n",
    "    print(f\"Before: {dynamic_notebook}\")\n",
    "    print(f\"Plan: {plan}\")\n",
    "    llm_output =   (\n",
    "        llm.as_structured_llm(IFOutput)\n",
    "        .complete(information_check_prompt.format(subqueries=subqueries, original_question=original_question, dynamic_notebook=dynamic_notebook, plan=plan))\n",
    "        .raw\n",
    "    )\n",
    "    print(f\"After: {llm_output.dynamic_notebook}\")\n",
    "    print(f\"New Plan: {llm_output.modified_plan}\")\n",
    "    return {'dynamic_notebook': llm_output.dynamic_notebook, 'modified_plan': llm_output.modified_plan}\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ed70406c-c95b-4e29-a863-cc29c14a89c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_answer_system = \"\"\"You are a highly intelligent assistant trained to provide concise and accurate answers. You will be given a context and a user question. Your task is to analyze the context and answer the user question based on the information provided in the context. If the context lacks sufficient information to answer the question, inform the user and suggest what additional details are needed.\n",
    "\n",
    "Focus solely on the context to form your response. Avoid making assumptions or using external knowledge unless explicitly stated in the context.\n",
    "Ensure the final answer is clear, relevant, and directly addresses the user’s question.\n",
    "If the question is ambiguous, ask clarifying questions to ensure accuracy before proceeding.\"\"\"\n",
    "\n",
    "final_answer_user = \"\"\"\n",
    "Based on this context:\n",
    "{context}\n",
    "\n",
    "Answer the following question:\n",
    "<question>\n",
    "{question}\n",
    "</question>\n",
    "\n",
    "Provide your answer based on the context above explain your reasoning.\n",
    "If clarification or additional information is needed, explain why and specify what is required.\n",
    "\"\"\"\n",
    "\n",
    "final_answer_msgs = [\n",
    "    (\n",
    "        \"system\",\n",
    "        final_answer_system,\n",
    "    ),\n",
    "    (\"user\", final_answer_user),\n",
    "]\n",
    "\n",
    "final_answer_prompt = ChatPromptTemplate.from_messages(final_answer_msgs)\n",
    "\n",
    "async def generate_final_answer(question, context):\n",
    "    resp = await llm_fast.achat(final_answer_prompt.format_messages(question=question, context=context))\n",
    "    return resp.message.content                 \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0ee5da9c-679f-4dd5-b99f-2d63334fd8c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_INFORMATION_CHECKS = 3\n",
    "\n",
    "class GenerateCypher(Event):\n",
    "    subquery: str\n",
    "    \n",
    "class ValidateCypher(Event):\n",
    "    subquery: str\n",
    "    generated_cypher: str\n",
    "\n",
    "class CorrectCypher(Event):\n",
    "    cypher: str\n",
    "    subquery: str\n",
    "    errors: List[str]\n",
    "\n",
    "class ExecuteCypher(Event):\n",
    "    validated_cypher: str\n",
    "    subquery: str\n",
    "\n",
    "class InformationCheck(Event):\n",
    "    cypher: str\n",
    "    subquery: str\n",
    "    database_output: list\n",
    "    \n",
    "class GenerateFinalAnswer(Event):\n",
    "    context: str\n",
    "\n",
    "class ConcurrentFlow(Workflow):\n",
    "    @step\n",
    "    async def start(self, ctx: Context, ev: StartEvent) -> GenerateCypher | GenerateFinalAnswer:\n",
    "        original_question = ev.input\n",
    "        await ctx.set(\"original_question\", original_question) # So we don't need to pass the question all the time\n",
    "        await ctx.set(\"information_checks\", 0) # Current number of information check steps\n",
    "        await ctx.set(\"dynamic_notebook\", \"\") # Current knowledge\n",
    "        await ctx.set(\"subqueries_cypher_history\", {}) # History of which queries were executed\n",
    "        guardrails_output = guardrails_step(original_question)\n",
    "        if guardrails_output.get(\"next_event\") == \"generate_final_answer\":\n",
    "            context = \"The question is not about movies or cast, so I cannot answer the question\"\n",
    "            return GenerateFinalAnswer(context=context)\n",
    "\n",
    "        # store in global context \n",
    "        subqueries = guardrails_output[\"arguments\"].get(\"plan\")\n",
    "        await ctx.set(\"count_of_subqueries\", len(subqueries[0])) #we use this in ctx.collect()\n",
    "        await ctx.set(\"plan\", subqueries[1:]) #Later plan after we finish current step\n",
    "        # Send events in the current step of the plan\n",
    "        for subquery in subqueries[0]:\n",
    "            print(subquery)\n",
    "            ctx.send_event(GenerateCypher(subquery=subquery))\n",
    "\n",
    "\n",
    "    @step(num_workers=4)\n",
    "    async def generate_cypher_step(self, ctx: Context, ev: GenerateCypher) -> ValidateCypher:\n",
    "        #print(\"Running generate_cypher \", ev.subquery)\n",
    "        generated_cypher = await generate_cypher(ev.subquery)\n",
    "        return ValidateCypher(subquery=ev.subquery, generated_cypher=generated_cypher)\n",
    "\n",
    "    @step(num_workers=4)\n",
    "    async def validate_cypher_step(self, ctx: Context, ev: ValidateCypher) -> GenerateFinalAnswer | ExecuteCypher | CorrectCypher:\n",
    "        #print(\"Running validate_cypher \", ev)\n",
    "        results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
    "        if results['next_action'] == \"end\": # DB value mapping\n",
    "            return GenerateFinalAnswer(context=str(results[\"mapping_errors\"]))\n",
    "        if results['next_action'] == \"execute_cypher\":\n",
    "            return ExecuteCypher(subquery=ev.subquery, validated_cypher=ev.generated_cypher)\n",
    "        if results['next_action'] == \"correct_cypher\":\n",
    "            return CorrectCypher(subquery=ev.subquery, cypher=ev.generated_cypher, errors=results['cypher_errors'])\n",
    "\n",
    "    @step(num_workers=4)\n",
    "    async def correct_cypher_step(self, ctx: Context, ev: CorrectCypher) -> ValidateCypher:\n",
    "        print(\"Running validate_cypher \", ev)\n",
    "        results = await correct_cypher(ev.subquery, ev.cypher, ev.errors)\n",
    "        return ValidateCypher(subquery=ev.subquery, generated_cypher=results)\n",
    "    \n",
    "    @step\n",
    "    async def execute_cypher_step(self, ctx: Context, ev: ExecuteCypher) -> InformationCheck:\n",
    "        try:\n",
    "            database_output = graph_store.structured_query(ev.validated_cypher)\n",
    "        except Exception as e: # Dividing by zero, etc... or timeout\n",
    "            database_output = e.message\n",
    "        return InformationCheck(subquery=ev.subquery, cypher=ev.validated_cypher, database_output=database_output)\n",
    "\n",
    "    @step\n",
    "    async def information_check_step(self, ctx: Context, ev: InformationCheck) -> GenerateCypher | GenerateFinalAnswer:\n",
    "        # wait until we receive all events\n",
    "        #print(\"Running information_check_step\", ev)\n",
    "        # retrieve from context\n",
    "        number_of_subqueries = await ctx.get(\"count_of_subqueries\")\n",
    "        result = ctx.collect_events(ev, [InformationCheck] * number_of_subqueries)\n",
    "        if result is None:\n",
    "            return None\n",
    "        # Add executed cypher statements to global state\n",
    "        subqueries_cypher_history = await ctx.get(\"subqueries_cypher_history\")\n",
    "        new_subqueries_cypher = {\n",
    "                item.subquery: {\n",
    "                    \"cypher\": item.cypher,\n",
    "                    \"database_output\": item.database_output\n",
    "                } for item in result\n",
    "            }\n",
    "        await ctx.set(\"subqueries_cypher_history\", {**subqueries_cypher_history, **new_subqueries_cypher})\n",
    "\n",
    "        original_question = await ctx.get(\"original_question\")\n",
    "        dynamic_notebook = await ctx.get(\"dynamic_notebook\")\n",
    "        plan = await ctx.get(\"plan\")\n",
    "\n",
    "        # Do the information check\n",
    "        \n",
    "        data = information_check(result, original_question, dynamic_notebook, plan)\n",
    "        # Get count of information checks done\n",
    "        information_checks = await ctx.get(\"information_checks\")\n",
    "        # Go fetch additional information if needed\n",
    "        if data.get(\"modified_plan\") and information_checks < MAX_INFORMATION_CHECKS:\n",
    "            await ctx.set(\"count_of_subqueries\", len(data['modified_plan'][0])) # this is used for ctx.collect()\n",
    "            await ctx.set(\"dynamic_notebook\", data[\"dynamic_notebook\"])\n",
    "            await ctx.set(\"plan\", data.get(\"modified_plan\")[1:])\n",
    "            await ctx.set(\"information_checks\", information_checks + 1)\n",
    "            for subquery in data[\"modified_plan\"][0]:\n",
    "                ctx.send_event(GenerateCypher(subquery=subquery))\n",
    "        else:\n",
    "            return GenerateFinalAnswer(context=data['dynamic_notebook'])\n",
    "\n",
    "    @step\n",
    "    async def final_answer(self, ctx: Context, ev: GenerateFinalAnswer) -> StopEvent:\n",
    "        original_question = await ctx.get(\"original_question\")\n",
    "        subqueries_cypher_history = await ctx.get(\"subqueries_cypher_history\")\n",
    "        # wait until we receive all events\n",
    "        print(\"Running final_answer \", ev)\n",
    "        resp = await generate_final_answer(original_question, ev.context)\n",
    "        return StopEvent(result={\"text\":resp, \"subqueries_cypher_history\": subqueries_cypher_history})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f455f31f-2a02-4e84-b75a-bdd951f147aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom llama_index.utils.workflow import (\\n    draw_all_possible_flows,\\n    draw_most_recent_execution,\\n)\\n\\ndraw_most_recent_execution(w, filename=\"joke_flow_recent.html\")\\ndraw_all_possible_flows(w, filename=\"joke_flow_recenst.html\")\\n'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "from llama_index.utils.workflow import (\n",
    "    draw_all_possible_flows,\n",
    "    draw_most_recent_execution,\n",
    ")\n",
    "\n",
    "draw_most_recent_execution(w, filename=\"joke_flow_recent.html\")\n",
    "draw_all_possible_flows(w, filename=\"joke_flow_recenst.html\")\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "521ae8dd-9011-4937-9374-fd3d8ad52cee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>schema</th>\n",
       "      <th>cypher</th>\n",
       "      <th>data_source</th>\n",
       "      <th>instance_id</th>\n",
       "      <th>database_reference_alias</th>\n",
       "      <th>question_embedding</th>\n",
       "      <th>text_answers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>List the first 3 movies that have been directe...</td>\n",
       "      <td>Node properties:\\n- **Movie**\\n  - `url`: STRI...</td>\n",
       "      <td>MATCH (p:Person)-[:ACTED_IN]-&gt;(m:Movie)&lt;-[:DIR...</td>\n",
       "      <td>neo4jLabs_synthetic_claudeopus</td>\n",
       "      <td>instance_id_35354</td>\n",
       "      <td>neo4jlabs_demo_db_recommendations</td>\n",
       "      <td>[-0.04787346348166466, 0.02376137115061283, 0....</td>\n",
       "      <td>The first three movies that have been directed...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>List the first 5 directors who have directed a...</td>\n",
       "      <td>Node properties:\\n- **Movie**\\n  - `url`: STRI...</td>\n",
       "      <td>MATCH (d:Director)-[:DIRECTED]-&gt;(m:Movie) WHER...</td>\n",
       "      <td>neo4jLabs_synthetic_claudeopus</td>\n",
       "      <td>instance_id_35510</td>\n",
       "      <td>neo4jlabs_demo_db_recommendations</td>\n",
       "      <td>[-0.02948657236993313, 0.028692537918686867, 0...</td>\n",
       "      <td>The first 5 directors who have directed a movi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            question  \\\n",
       "0  List the first 3 movies that have been directe...   \n",
       "1  List the first 5 directors who have directed a...   \n",
       "\n",
       "                                              schema  \\\n",
       "0  Node properties:\\n- **Movie**\\n  - `url`: STRI...   \n",
       "1  Node properties:\\n- **Movie**\\n  - `url`: STRI...   \n",
       "\n",
       "                                              cypher  \\\n",
       "0  MATCH (p:Person)-[:ACTED_IN]->(m:Movie)<-[:DIR...   \n",
       "1  MATCH (d:Director)-[:DIRECTED]->(m:Movie) WHER...   \n",
       "\n",
       "                      data_source        instance_id  \\\n",
       "0  neo4jLabs_synthetic_claudeopus  instance_id_35354   \n",
       "1  neo4jLabs_synthetic_claudeopus  instance_id_35510   \n",
       "\n",
       "            database_reference_alias  \\\n",
       "0  neo4jlabs_demo_db_recommendations   \n",
       "1  neo4jlabs_demo_db_recommendations   \n",
       "\n",
       "                                  question_embedding  \\\n",
       "0  [-0.04787346348166466, 0.02376137115061283, 0....   \n",
       "1  [-0.02948657236993313, 0.028692537918686867, 0...   \n",
       "\n",
       "                                        text_answers  \n",
       "0  The first three movies that have been directed...  \n",
       "1  The first 5 directors who have directed a movi...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# URL of the Parquet file\n",
    "url = \"https://huggingface.co/datasets/tomasonjo/text2cypher_demo_test/resolve/main/test.parquet\"\n",
    "\n",
    "# Read the Parquet file directly from the URL\n",
    "df = pd.read_parquet(url)\n",
    "\n",
    "# Display the first few rows\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2000dd85-f6be-46a6-8d04-f45750a3f698",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 at 2024-12-19 11:48:04.085334\n",
      "##############################\n",
      "List all movies directed by actors.\n",
      "Before: \n",
      "Plan: [['List the release date of each movie from the previous query.']]\n",
      "After: The movie 'Mighty Aphrodite' is directed by an actor.\n",
      "New Plan: [[\"List the release date of the movie 'Mighty Aphrodite'.\"], ['List the first 3 movies that have been directed by actors.']]\n",
      "Before: The movie 'Mighty Aphrodite' is directed by an actor.\n",
      "Plan: [['List the first 3 movies that have been directed by actors.']]\n",
      "After: The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10.\n",
      "New Plan: [['List the first 3 movies that have been directed by actors.']]\n",
      "Before: The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10.\n",
      "Plan: []\n",
      "After: The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10. We have found 1 movie directed by an actor, but we need 3.\n",
      "New Plan: [['List the next 2 movies that have been directed by actors.']]\n",
      "Before: The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10. We have found 1 movie directed by an actor, but we need 3.\n",
      "Plan: []\n",
      "After: The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10. We have found 1 movie directed by an actor, but we need 3.\n",
      "New Plan: [['List the next 2 movies that have been directed by actors.']]\n",
      "Running final_answer  context=\"The movie 'Mighty Aphrodite' is directed by an actor. The release date of the movie 'Mighty Aphrodite' is 1995-11-10. We have found 1 movie directed by an actor, but we need 3.\"\n",
      "1 at 2024-12-19 11:48:32.611549\n",
      "##############################\n",
      "List all movies with an imdbRating of 9 or higher.\n",
      "Before: \n",
      "Plan: [['For each movie identified in the previous step, list the director(s).']]\n",
      "After: The movie 'Godfather: Part II, The' has an imdbRating of 9 or higher.\n",
      "New Plan: [[\"For the movie 'Godfather: Part II, The', list the director(s).\"], ['Limit the results to the first 5 directors.']]\n",
      "Before: The movie 'Godfather: Part II, The' has an imdbRating of 9 or higher.\n",
      "Plan: [['Limit the results to the first 5 directors.']]\n",
      "After: The movie 'Godfather: Part II, The' has an imdbRating of 9 or higher and is directed by Francis Ford Coppola.\n",
      "New Plan: [['Find all movies with an imdbRating of 9 or higher and their directors.'], ['Limit the results to the first 5 directors.']]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-f35fab2db11f', bound_args=<BoundArgumen...or higher.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x3527117c0>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-f35fab2db11f', bound_args=<BoundArgumen...or higher.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x3527117c0>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=9, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=9, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 at 2024-12-19 11:48:54.550165\n",
      "##############################\n",
      "List all directors and the languages of the movies they have directed.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "New Plan: [['List all directors and the languages of the movies they have directed.']]\n",
      "Before: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "Plan: []\n",
      "After: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "New Plan: [['List all directors and the languages of the movies they have directed.']]\n",
      "Before: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "Plan: []\n",
      "After: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "New Plan: [['List all directors and the languages of the movies they have directed.']]\n",
      "Before: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "Plan: []\n",
      "After: The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\n",
      "New Plan: [['List all directors and the languages of the movies they have directed.']]\n",
      "Running final_answer  context=\"The current notebook contains information about directors and the languages of the movies they have directed. However, it only contains information about one director, John Lasseter, and his movies are only in English. To answer the question 'Which three directors have directed movies in more than one language?', we need to gather information about more directors and their movies' languages.\"\n",
      "3 at 2024-12-19 11:49:28.319482\n",
      "##############################\n",
      "What are the names of movies that have a plot mentioning 'evil exterminator'?\n",
      "Before: \n",
      "Plan: [['What is the IMDb rating of each movie identified in the previous step?']]\n",
      "After: The movie 'Boxtrolls, The' has a plot mentioning 'evil exterminator'.\n",
      "New Plan: [[\"What is the IMDb rating of the movie 'Boxtrolls, The'?\"]]\n",
      "Before: The movie 'Boxtrolls, The' has a plot mentioning 'evil exterminator'.\n",
      "Plan: []\n",
      "After: The movie 'Boxtrolls, The' has a plot mentioning 'evil exterminator' and an IMDb rating of 6.8.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The movie 'Boxtrolls, The' has a plot mentioning 'evil exterminator' and an IMDb rating of 6.8.\"\n",
      "4 at 2024-12-19 11:49:48.592120\n",
      "##############################\n",
      "What are the budgets of all movies?\n",
      "What are the revenues of all movies?\n",
      "Before: \n",
      "Plan: []\n",
      "After: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3.\n",
      "New Plan: [['What are the budgets of all movies?', 'What are the revenues of all movies?'], ['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3.\n",
      "Plan: [['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "After: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\n",
      "New Plan: [['What are the budgets of all movies?', 'What are the revenues of all movies?'], ['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\n",
      "Plan: [['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "After: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\n",
      "New Plan: [['What are the budgets of all movies?', 'What are the revenues of all movies?'], ['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\n",
      "Plan: [['Calculate the budget to revenue ratio for each movie.', 'Sort the movies by the calculated ratio in descending order.', 'Select the top 3 movies.']]\n",
      "After: The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\n",
      "New Plan: [['What are the budgets of all movies?', 'What are the revenues of all movies?'], ['Calculate the budget to revenue ratio for each movie.'], ['Sort the movies by the calculated ratio in descending order.'], ['Select the top 3 movies.']]\n",
      "Running final_answer  context=\"The budget and revenue for the movie 'Toy Story' are available. Budget: 30000000, Revenue: 373554033. To answer the question 'List the top 3 movies with the highest budget to revenue ratio.', we need the budget and revenue for all movies, and then calculate the ratio and sort to find the top 3. Currently, we only have data for 'Toy Story'.\"\n",
      "5 at 2024-12-19 11:50:38.768611\n",
      "##############################\n",
      "What is the budget for each movie in the 'Science Fiction' genre?\n",
      "Running final_answer  context='[\"Could not find node in graph with label \\'Genre\\' where property \\'name\\' equals \\'Science Fiction\\'. Without this information, I cannot provide a complete answer to your question. If you meant something else, please rephrase your question or verify the specific name you\\'re asking about. Would you like to try with a different name value?\"]'\n",
      "6 at 2024-12-19 11:50:54.125375\n",
      "##############################\n",
      "List all movies with a runtime under 90 minutes.\n",
      "List all movies by worldwide revenue.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-91a2dfc1d4be', bound_args=<BoundArgumen...0 minutes.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x35552a380>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-91a2dfc1d4be', bound_args=<BoundArgumen...0 minutes.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x35552a380>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=90, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=90, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 at 2024-12-19 11:51:11.300529\n",
      "##############################\n",
      "Retrieve all user IDs and their corresponding movie ratings.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-e3c9368ea61d', bound_args=<BoundArgumen...to movies.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x356670fc0>)(<WorkflowHand...v/list_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-e3c9368ea61d', bound_args=<BoundArgumen...to movies.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x356670fc0>)(<WorkflowHand...v/list_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 79, in execute_cypher_step\n",
      "    return InformationCheck(subquery=ev.subquery, cypher=ev.validated_cypher, database_output=database_output)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/events.py\", line 74, in __init__\n",
      "    super().__init__(**fields)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 209, in __init__\n",
      "    validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)\n",
      "                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for InformationCheck\n",
      "database_output\n",
      "  Input should be a valid list [type=list_type, input_value='The transaction has been...with a longer timeout. ', input_type=str]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/list_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'execute_cypher_step': 1 validation error for InformationCheck\n",
      "database_output\n",
      "  Input should be a valid list [type=list_type, input_value='The transaction has been...with a longer timeout. ', input_type=str]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/list_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 at 2024-12-19 11:51:36.226635\n",
      "##############################\n",
      "List all actors born in France.\n",
      "Before: \n",
      "Plan: [['List all movies where each actor from the previous list was the main actor.'], ['List the release date of each movie from the previous list.']]\n",
      "After: The actor François Lallement was born in France. We need to find the movies where he was the main actor and then list the release dates of those movies to answer the question.\n",
      "New Plan: [[\"List all movies where the actor 'François Lallement' was the main actor.\"], ['List the release date of each movie from the previous list.'], ['Select the first 3 movies from the previous list.']]\n",
      "Before: The actor François Lallement was born in France. We need to find the movies where he was the main actor and then list the release dates of those movies to answer the question.\n",
      "Plan: [['List the release date of each movie from the previous list.'], ['Select the first 3 movies from the previous list.']]\n",
      "After: The actor François Lallement was born in France. We need to find the movies where he was the main actor and then list the release dates of those movies to answer the question. However, the subquery to list all movies where the actor 'François Lallement' was the main actor returned no results. Therefore, we cannot answer the question as we have no movies to work with.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The actor François Lallement was born in France. We need to find the movies where he was the main actor and then list the release dates of those movies to answer the question. However, the subquery to list all movies where the actor 'François Lallement' was the main actor returned no results. Therefore, we cannot answer the question as we have no movies to work with.\"\n",
      "9 at 2024-12-19 11:51:57.719388\n",
      "##############################\n",
      "List all countries that have produced movies in the database.\n",
      "List the number of movies produced by each country in the database.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The database contains movies produced in the USA. The USA has produced 6333 movies.\n",
      "New Plan: None\n",
      "Running final_answer  context='The database contains movies produced in the USA. The USA has produced 6333 movies.'\n",
      "10 at 2024-12-19 11:52:15.654591\n",
      "##############################\n",
      "List all movies and their associated actors.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient.\n",
      "New Plan: [['List all movies and their associated actors.'], ['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient.\n",
      "Plan: [['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "After: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\n",
      "New Plan: [['List all movies and their associated actors.'], ['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\n",
      "Plan: [['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "After: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\n",
      "New Plan: [['List all movies and their associated actors.'], ['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "Before: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\n",
      "Plan: [['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "After: The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\n",
      "New Plan: [['List all movies and their associated actors.'], ['Count the number of actors for each movie.', 'Sort movies by the number of associated actors in descending order.', 'Select the top 3 movies.']]\n",
      "Running final_answer  context=\"The subquery provided a movie title 'Toy Story' and its associated actors ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles']. To answer the question 'What are the first 3 movies with the most number of associated actors?', we need to retrieve this information for all movies and then count the number of actors for each movie. The current information is insufficient. We need to retrieve the movie titles and their associated actors for all movies.\"\n",
      "11 at 2024-12-19 11:52:51.275021\n",
      "##############################\n",
      "What are the names of all directors born in the USA?\n",
      "Before: \n",
      "Plan: [['For each director identified in the previous step, what are the names of their first 5 movies?']]\n",
      "After: The director Jessie Nelson is born in the USA.\n",
      "New Plan: [['What are the names of the first 5 movies directed by Jessie Nelson?']]\n",
      "Before: The director Jessie Nelson is born in the USA.\n",
      "Plan: []\n",
      "After: The director Jessie Nelson is born in the USA. The first movie directed by Jessie Nelson is 'Corrina, Corrina'.\n",
      "New Plan: [['What are the names of the first 5 movies directed by directors born in the USA, excluding movies directed by Jessie Nelson?']]\n",
      "Before: The director Jessie Nelson is born in the USA. The first movie directed by Jessie Nelson is 'Corrina, Corrina'.\n",
      "Plan: []\n",
      "After: The director Jessie Nelson is born in the USA. The first movie directed by Jessie Nelson is 'Corrina, Corrina'. The first 5 movies directed by directors born in the USA, excluding movies directed by Jessie Nelson, include 'Threesome'.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The director Jessie Nelson is born in the USA. The first movie directed by Jessie Nelson is 'Corrina, Corrina'. The first 5 movies directed by directors born in the USA, excluding movies directed by Jessie Nelson, include 'Threesome'.\"\n",
      "12 at 2024-12-19 11:53:17.961473\n",
      "##############################\n",
      "List all actors and the movies they have acted in that are classified as 'Adventure' genre.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question.\n",
      "New Plan: [[\"List all actors and the movies they have acted in that are classified as 'Adventure' genre.\"], ['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "Before: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question.\n",
      "Plan: [['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "After: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\n",
      "New Plan: [[\"List all actors and the movies they have acted in that are classified as 'Adventure' genre.\"], ['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "Before: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\n",
      "Plan: [['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "After: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\n",
      "New Plan: [[\"List all actors and the movies they have acted in that are classified as 'Adventure' genre.\"], ['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "Before: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\n",
      "Plan: [['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "After: The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\n",
      "New Plan: [[\"List all actors and the movies they have acted in that are classified as 'Adventure' genre.\"], ['Count the number of adventure movies each actor has acted in.'], ['Sort the actors by the number of adventure movies in descending order.'], ['Select the top 5 actors from the sorted list.']]\n",
      "Running final_answer  context=\"The current dynamic notebook contains information about actors and the movies they have acted in that are classified as 'Adventure' genre. For example, 'Jared Harris' acted in 'Boxtrolls, The'. However, this is only one entry, and we need to find all actors and their adventure movies to determine the top 5 actors by the number of movies acted in the 'Adventure' genre. The current information is insufficient to answer the question. We need to retrieve all actors and their adventure movies.\"\n",
      "13 at 2024-12-19 11:53:53.885257\n",
      "##############################\n",
      "What are all the genres listed in the dataset?\n",
      "What is the IMDb rating for each movie in the dataset?\n",
      "Before: \n",
      "Plan: [['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "After: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "New Plan: [['What are all the genres listed in the dataset?', 'What is the IMDb rating for each movie in the dataset?', 'For each movie, what are its associated genres?'], ['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "Before: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "Plan: [['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "After: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "New Plan: [['What are all the genres listed in the dataset?', 'What is the IMDb rating for each movie in the dataset?', 'For each movie, what are its associated genres?'], ['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "Before: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "Plan: [['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "After: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "New Plan: [['What are all the genres listed in the dataset?', 'What is the IMDb rating for each movie in the dataset?', 'For each movie, what are its associated genres?'], ['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "Before: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "Plan: [['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "After: The current notebook contains the following information:\n",
      "- Genres: Adventure\n",
      "- Movie: Toy Story, IMDb Rating: 8.3\n",
      "\n",
      "This information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\n",
      "New Plan: [['What are all the genres listed in the dataset?', 'What is the IMDb rating for each movie in the dataset?', 'For each movie, what are its associated genres?'], ['For each genre, what are the IMDb ratings of all movies belonging to that genre?']]\n",
      "Running final_answer  context=\"The current notebook contains the following information:\\n- Genres: Adventure\\n- Movie: Toy Story, IMDb Rating: 8.3\\n\\nThis information is insufficient to answer the question 'Which three genres have the lowest average IMDb rating?' because we only have one genre and one movie rating. We need to know all genres and the ratings of all movies within each genre to calculate average ratings.\"\n",
      "14 at 2024-12-19 11:54:46.741931\n",
      "##############################\n",
      "List all movies released before 2000 with their respective budgets.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-32343aac4233', bound_args=<BoundArgumen...fore 2000.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x357b6ea80>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-32343aac4233', bound_args=<BoundArgumen...fore 2000.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x357b6ea80>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=2000, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=2000, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 at 2024-12-19 11:55:00.509371\n",
      "##############################\n",
      "List all movies that have been rated after 2015.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-624505d9a438', bound_args=<BoundArgumen...fter 2015.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x356863180>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-624505d9a438', bound_args=<BoundArgumen...fter 2015.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x356863180>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=1420070400, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=1420070400, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 at 2024-12-19 11:55:14.939884\n",
      "##############################\n",
      "What are the release years of all movies in the database?\n",
      "What is the current year?\n",
      "Before: \n",
      "Plan: [['Which movies were released in the last 5 years based on the release years and current year?', 'What is the imdbVotes for each movie in the database?'], ['Which movies from the list of movies released in the last 5 years have the highest imdbVotes?']]\n",
      "After: The database contains movies released in 1995. The current year is 2024. The remaining plan is to first find movies released in the last 5 years and their imdbVotes, then find the 3 movies with the highest imdbVotes.\n",
      "New Plan: [['Which movies were released in the last 5 years based on the release years and current year?', 'What is the imdbVotes for each movie in the database?'], ['Which movies from the list of movies released in the last 5 years have the highest imdbVotes?']]\n",
      "Before: The database contains movies released in 1995. The current year is 2024. The remaining plan is to first find movies released in the last 5 years and their imdbVotes, then find the 3 movies with the highest imdbVotes.\n",
      "Plan: [['Which movies from the list of movies released in the last 5 years have the highest imdbVotes?']]\n",
      "After: The database contains movies released in 1995. The current year is 2024. Subquery to find movies released in the last 5 years returned no results. The imdbVotes for 'Toy Story' is 591836. The original question asks for the 3 movies released in the last 5 years with the highest imdbVotes. Since the subquery to find movies released in the last 5 years returned no results, the question cannot be answered.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The database contains movies released in 1995. The current year is 2024. Subquery to find movies released in the last 5 years returned no results. The imdbVotes for 'Toy Story' is 591836. The original question asks for the 3 movies released in the last 5 years with the highest imdbVotes. Since the subquery to find movies released in the last 5 years returned no results, the question cannot be answered.\"\n",
      "17 at 2024-12-19 11:55:42.386196\n",
      "##############################\n",
      "List all movies with a runtime over 180 minutes.\n",
      "List all directors.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-e03bdb6b1c44', bound_args=<BoundArgumen...0 minutes?'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37a676540>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-e03bdb6b1c44', bound_args=<BoundArgumen...0 minutes?'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37a676540>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=180, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=180, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 at 2024-12-19 11:55:59.340647\n",
      "##############################\n",
      "List all directors and the countries where they have directed movies.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question.\n",
      "New Plan: [['List all directors and the countries where they have directed movies.'], ['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "Before: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question.\n",
      "Plan: [['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "After: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries.\n",
      "New Plan: [['List all directors and the countries where they have directed movies.'], ['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "Before: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries.\n",
      "Plan: [['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "After: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries. The existing subquery only returned one director and country, which is not enough to determine which directors have directed movies in more than three countries.\n",
      "New Plan: [['List all directors and the countries where they have directed movies.'], ['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "Before: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries. The existing subquery only returned one director and country, which is not enough to determine which directors have directed movies in more than three countries.\n",
      "Plan: [['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "After: The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries. The existing subquery only returned one director and country, which is not enough to determine which directors have directed movies in more than three countries.\n",
      "New Plan: [['List all directors and the countries where they have directed movies.'], ['Group the results by director and count the number of distinct countries they have directed movies in.'], ['Filter the grouped results to include only directors who have directed movies in more than three countries.'], ['List the top 5 directors from the filtered results.']]\n",
      "Running final_answer  context=\"The current notebook contains information about directors and the countries where they have directed movies. For example, 'John Lasseter' has directed movies in 'USA'. However, this is only one entry, and we need to find directors who have directed movies in more than three different countries. The current information is insufficient to answer the question. We need to fetch more data about directors and their countries. The existing subquery only returned one director and country, which is not enough to determine which directors have directed movies in more than three countries.\"\n",
      "19 at 2024-12-19 11:56:34.391083\n",
      "##############################\n",
      "List the first 5 movies that have been directed by women.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The subquery to list the first 5 movies directed by women returned no results. Therefore, there is no information available to answer the question. The task is unsolvable due to missing data.\n",
      "New Plan: None\n",
      "Running final_answer  context='The subquery to list the first 5 movies directed by women returned no results. Therefore, there is no information available to answer the question. The task is unsolvable due to missing data.'\n",
      "20 at 2024-12-19 11:56:50.031100\n",
      "##############################\n",
      "List all movies and the number of actors participating in each movie.\n",
      "Before: \n",
      "Plan: []\n",
      "After: The movie 'Toy Story' has 4 actors participating. We need to compare this with other movies to find the one with the largest number of actors.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The movie 'Toy Story' has 4 actors participating. We need to compare this with other movies to find the one with the largest number of actors.\"\n",
      "21 at 2024-12-19 11:57:07.173720\n",
      "##############################\n",
      "List all actors who were born in the USA.\n",
      "Before: \n",
      "Plan: [['List all movies that are classified as comedy genre.'], ['For each actor from the first query, list all movies they have acted in.']]\n",
      "After: Actors born in the USA: {'a.name': 'Dodie Marshall'}\n",
      "New Plan: [['List all movies that are classified as comedy genre.'], ['For each actor from the dynamic notebook, list all movies they have acted in.']]\n",
      "Before: Actors born in the USA: {'a.name': 'Dodie Marshall'}\n",
      "Plan: [['For each actor from the dynamic notebook, list all movies they have acted in.']]\n",
      "After: Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}.\n",
      "New Plan: [['For each actor from the dynamic notebook, list all movies they have acted in.'], ['For each movie from the dynamic notebook, list all actors who have acted in it.'], ['From the results of the previous two steps, identify movies that are both comedies and have actors who were born in the USA.']]\n",
      "Before: Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}.\n",
      "Plan: [['For each movie from the dynamic notebook, list all actors who have acted in it.'], ['From the results of the previous two steps, identify movies that are both comedies and have actors who were born in the USA.']]\n",
      "After: Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}. Actor Jim Varney acted in movies: ['Toy Story', 'Beverly Hillbillies, The', '3 Ninjas: High Noon On Mega Mountain', 'Ernest Goes to Camp', 'Ernest Saves Christmas', 'Ernest Scared Stupid'].\n",
      "New Plan: [['For each movie from the dynamic notebook, list all actors who have acted in it.'], ['From the results of the previous two steps, identify movies that are both comedies and have actors who were born in the USA.']]\n",
      "Before: Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}. Actor Jim Varney acted in movies: ['Toy Story', 'Beverly Hillbillies, The', '3 Ninjas: High Noon On Mega Mountain', 'Ernest Goes to Camp', 'Ernest Saves Christmas', 'Ernest Scared Stupid'].\n",
      "Plan: [['From the results of the previous two steps, identify movies that are both comedies and have actors who were born in the USA.']]\n",
      "After: Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}. Actor Jim Varney acted in movies: ['Toy Story', 'Beverly Hillbillies, The', '3 Ninjas: High Noon On Mega Mountain', 'Ernest Goes to Camp', 'Ernest Saves Christmas', 'Ernest Scared Stupid']. Movie 'Toy Story' has actors: ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles'].\n",
      "New Plan: [['For each actor in the dynamic notebook, find their birth country.', 'For each movie in the dynamic notebook, find its genre.'], ['From the results of the previous two steps, identify movies that are both comedies and have actors who were born in the USA.']]\n",
      "Running final_answer  context=\"Actors born in the USA: {'a.name': 'Dodie Marshall'}. Movies with comedy genre: {'m.title': 'Two Night Stand'}. Actor Jim Varney acted in movies: ['Toy Story', 'Beverly Hillbillies, The', '3 Ninjas: High Noon On Mega Mountain', 'Ernest Goes to Camp', 'Ernest Saves Christmas', 'Ernest Scared Stupid']. Movie 'Toy Story' has actors: ['Jim Varney', 'Tim Allen', 'Tom Hanks', 'Don Rickles'].\"\n",
      "22 at 2024-12-19 11:57:44.962729\n",
      "##############################\n",
      "What are all the movies in the database?\n",
      "What are all the genres in the database?\n",
      "Before: \n",
      "Plan: [['For each movie, what are its associated genres?']]\n",
      "After: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. However, the association between movies and genres is missing. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie.\n",
      "New Plan: [['For each movie, what are its associated genres?'], ['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "Before: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. However, the association between movies and genres is missing. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie.\n",
      "Plan: [['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "After: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie.\n",
      "New Plan: [['For each movie, what are its associated genres?'], ['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "Before: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie.\n",
      "Plan: [['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "After: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie. We have a partial result for one movie. We need to get the genres for all movies to answer the question.\n",
      "New Plan: [['For each movie, what are its associated genres?'], ['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "Before: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie. We have a partial result for one movie. We need to get the genres for all movies to answer the question.\n",
      "Plan: [['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "After: The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie. We have a partial result for one movie. We need to get the genres for all movies to answer the question.\n",
      "New Plan: [['For each movie, what are its associated genres?'], ['Count the number of distinct genres for each movie.'], ['Rank movies by the count of distinct genres in descending order.'], ['Select the top 5 movies from the ranked list.']]\n",
      "Running final_answer  context=\"The database contains at least one movie titled 'Toy Story' and at least one genre named 'Adventure'. The association between movies and genres is now partially known, with 'Boxtrolls, The' having genres ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy']. The original question asks for the top 5 movies with the most distinct genres, which requires knowing the genres associated with each movie. We have a partial result for one movie. We need to get the genres for all movies to answer the question.\"\n",
      "23 at 2024-12-19 11:58:24.059892\n",
      "##############################\n",
      "List all movies with a budget less than 50 million dollars.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-83b33db66ded', bound_args=<BoundArgumen...n dollars.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37a674ac0>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-83b33db66ded', bound_args=<BoundArgumen...n dollars.'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37a674ac0>)(<WorkflowHand...string_type\")>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 247, in _task\n",
      "    new_ev = await instrumented_step(**kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 367, in async_wrapper\n",
      "    result = await func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/7964307.py\", line 59, in validate_cypher_step\n",
      "    results = validate_cypher(ev.subquery, ev.generated_cypher)\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/var/folders/87/hnv4nyfj1bl8h65l0mp7xrx40000gp/T/ipykernel_65096/793065339.py\", line 19, in validate_cypher\n",
      "    .complete(validate_cypher_prompt.format(cypher=cypher))\n",
      "     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 431, in wrapped_llm_predict\n",
      "    f_return_val = f(_self, *args, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 107, in complete\n",
      "    return complete_fn(prompt, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/base/llms/generic_utils.py\", line 173, in wrapper\n",
      "    chat_response = func(messages, **kwargs)\n",
      "                    ^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/callbacks.py\", line 173, in wrapped_llm_chat\n",
      "    f_return_val = f(_self, messages, **kwargs)\n",
      "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/structured_llm.py\", line 75, in chat\n",
      "    output = self.llm.structured_predict(\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/llms/llm.py\", line 374, in structured_predict\n",
      "    result = program(llm_kwargs=llm_kwargs, **prompt_args)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/program/llm_program.py\", line 103, in __call__\n",
      "    output = self._output_parser.parse(raw_output)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 321, in wrapper\n",
      "    result = func(*args, **kwargs)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/output_parsers/pydantic.py\", line 62, in parse\n",
      "    return self._output_cls.model_validate_json(json_str)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/pydantic/main.py\", line 622, in model_validate_json\n",
      "    return cls.__pydantic_validator__.validate_json(json_data, strict=strict, context=context)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "pydantic_core._pydantic_core.ValidationError: 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=50000000, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 285, in handle_future_result\n",
      "    raise exception\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 438, in _run_workflow\n",
      "    raise exception_raised\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/workflow/workflow.py\", line 253, in _task\n",
      "    raise WorkflowRuntimeError(\n",
      "llama_index.core.workflow.errors.WorkflowRuntimeError: Error in step 'validate_cypher_step': 1 validation error for ValidateCypherOutput\n",
      "filters.0.property_value\n",
      "  Input should be a valid string [type=string_type, input_value=50000000, input_type=int]\n",
      "    For further information visit https://errors.pydantic.dev/2.9/v/string_type\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 at 2024-12-19 11:58:38.241576\n",
      "##############################\n",
      "What are the release dates of all movies in the USA?\n",
      "What are the titles of all movies released in the USA?\n",
      "Before: \n",
      "Plan: []\n",
      "After: The dynamic notebook contains the following information:\n",
      "- The movie 'Toy Story' was released in the USA on '1995-11-22'.\n",
      "New Plan: None\n",
      "Running final_answer  context=\"The dynamic notebook contains the following information:\\n- The movie 'Toy Story' was released in the USA on '1995-11-22'.\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in callback Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-f9deb3eb6fa3', bound_args=<BoundArgumen...n the USA?'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37c288b80>)(<WorkflowHandler cancelled>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273\n",
      "handle: <Handle Dispatcher.span.<locals>.wrapper.<locals>.handle_future_result(span_id='Workflow.run...-f9deb3eb6fa3', bound_args=<BoundArgumen...n the USA?'})>, instance=<__main__.Con...t 0x352711490>, context=<_contextvars...t 0x37c288b80>)(<WorkflowHandler cancelled>) at /Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py:273>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/asyncio/events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/Users/tomazbratanic/anaconda3/lib/python3.11/site-packages/llama_index/core/instrumentation/dispatcher.py\", line 283, in handle_future_result\n",
      "    exception = future.exception()\n",
      "                ^^^^^^^^^^^^^^^^^^\n",
      "asyncio.exceptions.CancelledError\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25 at 2024-12-19 11:58:58.176535\n",
      "##############################\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import time\n",
    "w = ConcurrentFlow(timeout=120, verbose=False)\n",
    "answers = []\n",
    "times = []\n",
    "\n",
    "# Add latency\n",
    "for i, row in df.iterrows():\n",
    "    print(f\"{i} at {datetime.now()}\")\n",
    "    time.sleep(10)\n",
    "    question = row['question']\n",
    "    print(\"#\" * 30) \n",
    "    start = datetime.now()\n",
    "    try:\n",
    "        result = await w.run(input=question)\n",
    "        answers.append(result)\n",
    "        times.append(start - datetime.now())\n",
    "    except:\n",
    "         answers.append(None)\n",
    "         times.append(start - datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec20d27c-981a-48f4-a901-6418acec881e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['answers'] = answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9592def-5d5b-47aa-8ada-9038b1acd8f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['times'] = times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09cc905f-744e-413e-ae09-d296a13648e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64d2f353-1db4-4e21-8a06-3025a9433a86",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"gpt-4.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49bdf3ca-4552-49f4-9ca3-ee63e86679d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import pandas as pd\n",
    "#df = pd.read_csv('gpt-4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d44cc644-a84e-4077-b9a1-a5b2cbe76eb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "questions = df['question'].to_list()\n",
    "            \n",
    "ground_truths = [str(el) for el in df['text_answers'].to_list()]\n",
    "\n",
    "answers = [str(el) for el in df['answers'].to_list()]\n",
    "contexts = [None] * len(questions)\n",
    "\n",
    "# To dict\n",
    "data = {\n",
    "    \"question\": questions,\n",
    "    \"answer\": answers,\n",
    "    \"contexts\": contexts,\n",
    "    \"ground_truths\": ground_truths\n",
    "}\n",
    "\n",
    "# Convert dict to dataset\n",
    "dataset = Dataset.from_dict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "937995e4-4075-4640-a5ad-1328b0cd5887",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "203"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "6593d3b7-dce6-415e-9220-caeb798e0b6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "105"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([len(el) for el in answers if len(el) > 1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "a4a84181-1089-4f97-8745-0d7ebc841f7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([len(el) for el in answers if len(el) == 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "305db1ff-7d4e-4b69-8953-30dc3bd768ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom ragas.llms import LlamaIndexLLMWrapper\\n\\nevaluator_llm = LlamaIndexLLMWrapper(OpenAI(model=\"gpt-4o\"))\\nmetrics = [\\n    Faithfulness(llm=evaluator_llm),\\n    AnswerRelevancy(llm=evaluator_llm),\\n    ContextPrecision(llm=evaluator_llm),\\n    ContextRecall(llm=evaluator_llm),\\n]\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "from ragas.llms import LlamaIndexLLMWrapper\n",
    "\n",
    "evaluator_llm = LlamaIndexLLMWrapper(OpenAI(model=\"gpt-4o\"))\n",
    "metrics = [\n",
    "    Faithfulness(llm=evaluator_llm),\n",
    "    AnswerRelevancy(llm=evaluator_llm),\n",
    "    ContextPrecision(llm=evaluator_llm),\n",
    "    ContextRecall(llm=evaluator_llm),\n",
    "]\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "50471fda-ad3a-47da-9266-8c5413b223a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53a8fd53648a4fba85ce8ca96d4b3d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/203 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception raised in Job[4]: BadRequestError(Error code: 400 - {'error': {'message': \"This model's maximum context length is 128000 tokens. However, your messages resulted in 359326 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}})\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[58], line 9\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m evaluate\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mragas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m      3\u001b[0m     faithfulness,\n\u001b[1;32m      4\u001b[0m     answer_relevancy,\n\u001b[1;32m      5\u001b[0m     context_recall,\n\u001b[1;32m      6\u001b[0m     context_precision,\n\u001b[1;32m      7\u001b[0m )\n\u001b[0;32m----> 9\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmetrics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m        \u001b[49m\u001b[43manswer_relevancy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m     16\u001b[0m df \u001b[38;5;241m=\u001b[39m result\u001b[38;5;241m.\u001b[39mto_pandas()\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/ragas/_analytics.py:205\u001b[0m, in \u001b[0;36mtrack_was_completed.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(func)\n\u001b[1;32m    203\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m t\u001b[38;5;241m.\u001b[39mAny:\n\u001b[1;32m    204\u001b[0m     track(IsCompleteEvent(event_type\u001b[38;5;241m=\u001b[39mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, is_completed\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m))\n\u001b[0;32m--> 205\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    206\u001b[0m     track(IsCompleteEvent(event_type\u001b[38;5;241m=\u001b[39mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, is_completed\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m))\n\u001b[1;32m    208\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/ragas/evaluation.py:308\u001b[0m, in \u001b[0;36mevaluate\u001b[0;34m(dataset, metrics, llm, embeddings, callbacks, in_ci, run_config, token_usage_parser, raise_exceptions, column_map, show_progress, batch_size, _run_id, _pbar)\u001b[0m\n\u001b[1;32m    305\u001b[0m scores: t\u001b[38;5;241m.\u001b[39mList[t\u001b[38;5;241m.\u001b[39mDict[\u001b[38;5;28mstr\u001b[39m, t\u001b[38;5;241m.\u001b[39mAny]] \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    306\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    307\u001b[0m     \u001b[38;5;66;03m# get the results\u001b[39;00m\n\u001b[0;32m--> 308\u001b[0m     results \u001b[38;5;241m=\u001b[39m \u001b[43mexecutor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresults\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    309\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m results \u001b[38;5;241m==\u001b[39m []:\n\u001b[1;32m    310\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m ExceptionInRunner()\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/ragas/executor.py:213\u001b[0m, in \u001b[0;36mExecutor.results\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    210\u001b[0m             nest_asyncio\u001b[38;5;241m.\u001b[39mapply()\n\u001b[1;32m    211\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_nest_asyncio_applied \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43masyncio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process_jobs\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    214\u001b[0m sorted_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m(results, key\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mlambda\u001b[39;00m x: x[\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m    215\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [r[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m sorted_results]\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/nest_asyncio.py:30\u001b[0m, in \u001b[0;36m_patch_asyncio.<locals>.run\u001b[0;34m(main, debug)\u001b[0m\n\u001b[1;32m     28\u001b[0m task \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mensure_future(main)\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 30\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_until_complete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m task\u001b[38;5;241m.\u001b[39mdone():\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/nest_asyncio.py:92\u001b[0m, in \u001b[0;36m_patch_loop.<locals>.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     90\u001b[0m     f\u001b[38;5;241m.\u001b[39m_log_destroy_pending \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m f\u001b[38;5;241m.\u001b[39mdone():\n\u001b[0;32m---> 92\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_once\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     93\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stopping:\n\u001b[1;32m     94\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/nest_asyncio.py:115\u001b[0m, in \u001b[0;36m_patch_loop.<locals>._run_once\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    108\u001b[0m     heappop(scheduled)\n\u001b[1;32m    110\u001b[0m timeout \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    111\u001b[0m     \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m ready \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stopping\n\u001b[1;32m    112\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mmin\u001b[39m(\u001b[38;5;28mmax\u001b[39m(\n\u001b[1;32m    113\u001b[0m         scheduled[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39m_when \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtime(), \u001b[38;5;241m0\u001b[39m), \u001b[38;5;241m86400\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m scheduled\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 115\u001b[0m event_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_selector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_process_events(event_list)\n\u001b[1;32m    118\u001b[0m end_time \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_clock_resolution\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/selectors.py:561\u001b[0m, in \u001b[0;36mKqueueSelector.select\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    559\u001b[0m ready \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    560\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 561\u001b[0m     kev_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_selector\u001b[38;5;241m.\u001b[39mcontrol(\u001b[38;5;28;01mNone\u001b[39;00m, max_ev, timeout)\n\u001b[1;32m    562\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mInterruptedError\u001b[39;00m:\n\u001b[1;32m    563\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m ready\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception raised in Job[1]: AssertionError()\n",
      "Exception raised in Job[19]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[20]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[21]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[22]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[23]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[24]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[25]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[26]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[27]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[28]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[29]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[30]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[31]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[32]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[33]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[34]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[35]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[36]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[37]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[38]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[39]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[40]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[41]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[42]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[43]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[44]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[45]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[46]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[47]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[48]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[49]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[50]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[51]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[52]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[53]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[54]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[55]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[56]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[57]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[58]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[59]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[60]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[61]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[62]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[63]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[64]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[65]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[66]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[67]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[68]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[69]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[70]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[71]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[72]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[73]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[74]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[75]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[76]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[77]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[78]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[79]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[80]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[81]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[82]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[83]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[84]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[85]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[17]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[86]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[87]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[88]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[89]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[90]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[91]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[92]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[93]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[18]: BadRequestError(Error code: 400 - {'error': {'message': \"This model's maximum context length is 128000 tokens. However, your messages resulted in 143119 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}})\n",
      "Exception raised in Job[94]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[95]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[96]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[97]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[98]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[99]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[100]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[101]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[102]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[103]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[104]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[105]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[106]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[107]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[108]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[109]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[110]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[111]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[112]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[113]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[114]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[115]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[116]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[117]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[118]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[119]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[120]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[121]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[122]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[123]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[124]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[125]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[126]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[127]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[128]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[129]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[130]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[131]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[132]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[133]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[134]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[135]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[136]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[137]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[138]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[139]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[140]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[141]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[142]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[143]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[144]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[145]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[146]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[147]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[148]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[149]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[150]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[151]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[152]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[153]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[154]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[155]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[156]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[157]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[158]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[159]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[160]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[161]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[162]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[163]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[164]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[165]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[166]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[167]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[168]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[169]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[170]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[171]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[172]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[173]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[174]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[175]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[176]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[177]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[178]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[179]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[180]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[181]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[182]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[183]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[184]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[185]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[186]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[187]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[188]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[189]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[190]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[191]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[192]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[193]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[194]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[195]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[196]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[197]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[198]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[199]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[200]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[201]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[202]: AssertionError(LLM is not set)\n",
      "Exception raised in Job[15]: AssertionError()\n",
      "Exception raised in Job[14]: AssertionError()\n",
      "Exception raised in Job[3]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[6]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[12]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[9]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[5]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[8]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[2]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[13]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[10]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[7]: AttributeError('NoneType' object has no attribute 'generate')\n",
      "Exception raised in Job[11]: AttributeError('NoneType' object has no attribute 'generate')\n"
     ]
    }
   ],
   "source": [
    "from ragas import evaluate\n",
    "from ragas.metrics import (\n",
    "    faithfulness,\n",
    "    answer_relevancy,\n",
    "    context_recall,\n",
    "    context_precision,\n",
    ")\n",
    "\n",
    "result = evaluate(\n",
    "    dataset = dataset, \n",
    "    metrics=[\n",
    "        answer_relevancy,\n",
    "    ],\n",
    ")\n",
    "\n",
    "df = result.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90dd2298-3633-4442-8bcb-eab0b1f11ec2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
